# -*- coding: utf-8 -*-
import re
import time
import requests
from bs4 import BeautifulSoup
import pandas as pd
import logging
from tqdm import tqdm  # 进度条支持，可选安装
import os
# 配置日志
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)

# 预编译正则表达

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Connection': 'keep-alive'
}
def fetch_url(url, retries=3):
    """获取URL内容，支持重试"""
    for attempt in range(retries):
        try:
            response = requests.get(url, headers=HEADERS, timeout=10)
            response.raise_for_status()  # 检查HTTP错误

            # 检查是否被反爬
            if "检测到有异常请求" in response.text:
                logging.warning("触发反爬机制，尝试等待...")
                time.sleep(10)
                continue

            return response.text

        except requests.exceptions.RequestException as e:
            logging.warning(f"请求失败 ({attempt + 1}/{retries}): {str(e)}")
            time.sleep(2)

    logging.error(f"无法获取URL: {url}")
    return None



def extract_product_info(html_content):
    """从HTML内容中提取所有产品信息"""
    soup = BeautifulSoup(html_content, 'html.parser')
    products = []

    # 查找所有包含特定样式的产品div
    product_divs = soup.find_all('div', style=lambda x: x and 'animation' in x and 'fadeIn' in x)

    for div in product_divs:
        # 提取产品名称
        name_tag = div.find('h5', class_='hdg')
        name = name_tag.text.strip() if name_tag else "未命名"

        # 提取产品类型
        type_tag = div.find('p', class_='productListItem')
        product_type = type_tag.find('span').text.strip() if type_tag and type_tag.find('span') else "未知类型"

        # 提取产品描述
        desc_tag = div.select_one('div.m14-bd > div:not(.u-bold) > p')
        description = desc_tag.text.strip() if desc_tag else "无描述"

        # 提取产品图片URL
        img_tag = div.find('img')
        image_url = img_tag['src'] if img_tag and 'src' in img_tag.attrs else ""

        # 提取详情链接
        link_tag = div.find('a', class_='btn')
        detail_url = link_tag['href'] if link_tag and 'href' in link_tag.attrs else ""

        products.append({
            "产品名称": name,
            "产品类型": product_type,
            "产品描述": description,
            "图片URL": image_url,
            "详情链接": detail_url
        })

    return products


baseurl = r"http://leuven-instruments.cn/?product/"
text = fetch_url(baseurl)
products = extract_product_info(text)

for  product in products:
    print(product)
    print(baseurl+product['详情链接'])

    c = fetch_url(baseurl+product['详情链接'])
    a = 10